package cn.doitedu

import org.apache.spark.SparkConf
import org.apache.spark.sql.{DataFrame, SparkSession}
import org.elasticsearch.spark.sql.EsSparkSQL

object HiveProfileTags2ElasticSearch {

  def main(args: Array[String]): Unit = {

    val conf = new SparkConf()
    conf.set("es.index.auto.create", "true")
    conf.set("es.nodes", "doitedu")
      .set("es.port", "9200")
      .set("es.nodes.wan.only", "true")

    val spark = SparkSession.builder()
      .appName("画像导出es")
      .master("local")
      .config(conf)
      .enableHiveSupport()
      .getOrCreate()

    // 读取hive中的标签数据
    val df: DataFrame = spark.read.table("dws.user_profile_01")

    // 然后用es的整合api，写出
    EsSparkSQL.saveToEs(df,"doit35",Map("es.mapping.id" -> "guid"))


    // 保存完后，可以用如下restapi检查
    // http://doitedu:9200/_cat/indices?pretty

    spark.close()
  }
}
